package cn.com.duiba.millionaire.center.spider.common;

import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.Downloader;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.proxy.Data5UProxyProvider;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.scheduler.RedisScheduler;

import java.util.ArrayList;
import java.util.List;

/**
 * Created by sunchangji on 2018/6/5.
 */
public class DuiBaSpider implements PageProcessor {
    private PageConsumer pageConsumer;

    private Spider enhanceSpider;

    private String name;

//    private static Set<Cookie> cookies;

    private DuiBaSpider(Builder builder){
        this.pageConsumer = builder.pageConsumer;
        this.name = builder.name;

        this.enhanceSpider = new EnhanceSpider(this);
        //待爬取队列放入redis中，分布式爬取
        if (builder.useCluster) {
            enhanceSpider.setScheduler(new RedisScheduler(SpiderRedis.getJedisPool()));
        } else {
            enhanceSpider.setScheduler(new QueueScheduler());
        }

//        Request request = new Request(builder.startURL[0]);
//        if (cookies != null){
//            for (Cookie c : cookies){
//                request.addCookie(c.getName(),c.getValue());
//            }
//        }
//        enhanceSpider.addRequest(request);
        enhanceSpider.addUrl(builder.startURL).thread(1);  //1个线程抓取
        if (builder.useProxy) {
            HttpClientDownloader downloader = new HttpClientDownloader();
            downloader.setProxyProvider(new Data5UProxyProvider("b323ab32e6ca1be6b803291943d47b62")); //这个是找仲磊买的代理IP
            enhanceSpider.setDownloader(downloader);
        } else {
            if (builder.downloader != null) {
                enhanceSpider.setDownloader(builder.downloader);
            } else {
                enhanceSpider.setDownloader(new HttpClientDownloader());
            }
        }

        if (!CollectionUtils.isEmpty(builder.pipelines)) {
            builder.pipelines.forEach(pipeline -> enhanceSpider.addPipeline(pipeline));
        }
    }


    @Override
    public void process(Page page) {
        this.pageConsumer.accept(page);
    }

    @Override
    public Site getSite() {
        return Site.me().setTimeOut(5000).setDomain(name)
                .setRetryTimes(3).setSleepTime(1000).setCycleRetryTimes(3)
                .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36");
    }

    public static Builder builder(){
        return new Builder();
    }


    public static class Builder{
        private String name;
        private String[] startURL;
        private boolean useProxy = false;
        private boolean useCluster = false;
        private PageConsumer pageConsumer;
        private Downloader downloader;
        private List<Pipeline> pipelines = new ArrayList<>();
//
//        public Builder setCookies(Set<Cookie> c){
//            cookies = c;
//            return this;
//        }

        public Builder setName(String name) {
            this.name = name;
            return this;
        }

        public Builder setStartURL(String... startURL) {
            this.startURL = startURL;
            return this;
        }

        public Builder setUseProxy(boolean useProxy) {
            this.useProxy = useProxy;
            return this;
        }

        public Builder setPageConsumer(PageConsumer pageConsumer) {
            this.pageConsumer = pageConsumer;
            return this;
        }

        public Builder setDownloader(Downloader downloader) {
            this.downloader = downloader;
            return this;
        }

        public Builder addPipeline(Pipeline pipeline) {
            this.pipelines.add(pipeline);
            return this;
        }

        public Builder setUseCluster(boolean useCluster){
            this.useCluster = useCluster;
            return this;
        }

        public DuiBaSpider build(){
            Preconditions.checkArgument(StringUtils.isNotEmpty(this.name),"爬虫任务名不能为空，任务名全局唯一");
            Preconditions.checkArgument(startURL.length != 0,"爬虫起始采集链接不能为空");
            Preconditions.checkArgument(this.pageConsumer != null,"爬虫解析规则不能为空");

            return new DuiBaSpider(this);
        }
    }

    public void run(){
        enhanceSpider.run();
    }

    public void stop(){
        enhanceSpider.stop();
    }
}
